# -*- coding: utf-8 -*-
'''
Created on 2017年4月7日

@author: ZhuJiahui
'''
import os
import time
from global_info.global_nlp import GlobalNLP
from file_utils.file_writer import quick_write_1d_to_text

def text_extraction(read_filename, write_directory):
    
    sg_tag_list = GlobalNLP.SG_TAG_LIST
    sg_tag_dict = GlobalNLP.SG_TAG_DICT
    null_str = GlobalNLP.NULL_STR
    
    total_selected_count = 0
    
    for i in range(len(sg_tag_list)):
        
        line_count = 1
        to_write = []
        this_doc_text = []
        
        with open(read_filename, 'rb') as f:
            for b_each_line in f:
                try:
                    each_line = b_each_line.decode("utf-8")
                except UnicodeDecodeError:
                    try:
                        each_line = b_each_line.decode("gbk", "ignore")
                    except:
                        each_line = null_str
                
                if (line_count % 6 == 2):
                    try:
                        this_url = each_line.strip()[5:-6].strip()
                    except:
                        this_url = null_str
                    
                    this_tag_flag = 0
                    for each in sg_tag_dict[sg_tag_list[i]]:
                        if each.strip() in this_url:
                            this_tag_flag = 1
                            break
                    if (this_tag_flag == 1):
                        this_doc_text.append(this_url)
                        this_doc_text.append(sg_tag_list[i])
                    
                elif (line_count % 6 == 4):
                    if (this_tag_flag == 1):
                        try:
                            this_title = each_line.strip()[14:-15].strip()
                        except:
                            this_title = null_str
                        this_doc_text.append(this_title)
                elif (line_count % 6 == 5):
                    if (this_tag_flag == 1):
                        try:
                            this_text = each_line.strip()[9:-10].strip()
                        except:
                            this_text = null_str
                        this_doc_text.append(this_text)
                elif (line_count % 6 == 0):
                    if (this_tag_flag == 1):
                        if (null_str not in this_doc_text) and (len(this_doc_text[-1].strip()) > 10):
                            to_write.append(GlobalNLP.CN_COLUMN_DELIMITER.join(this_doc_text))
                            total_selected_count += 1
                    this_doc_text = []
                    
                else:
                    pass
                
                line_count += 1
                    
        quick_write_1d_to_text(write_directory + "/" + str(i) + ".txt", to_write)       
    
    print('总计新闻条数： ', total_selected_count)
    

if __name__ == '__main__':
    
    start = time.clock()    
    now_directory = os.getcwd()
    root_directory = os.path.dirname(now_directory) + '/'
    read_filename = root_directory + u'dataset/sogou/news_tensite_xml.dat'
    write_directory = root_directory + u'dataset/sogou/text_category'
    
    if (not(os.path.exists(write_directory))):
        os.mkdir(write_directory)
    
    text_extraction(read_filename, write_directory)

    print('Total time %f seconds' % (time.clock() - start))
    
    
    